**this code is meant to be parallel to 6.databuild_listprice.do  As a result, a lot of this code is not actually necessary to compute the contract dated index.
*see also notes to 4.databuild_closingdated.do

local msamacro="la pho seattle dc denver sandiego chi sanfran vegas"

set more off
foreach j in `msamacro' {
use temp11e_`j'_r2,clear
gen trash="`j'"
gen thresh=1
replace thresh=13 if trash=="pho"
replace thresh=25 if trash=="seattle"

sort lowaddress timeid
gen chnglist=ln(price)-ln(price[_n-1]) if lowaddress==lowaddress[_n-1] 
gen timediff=timeid-timeid[_n-1] if lowaddress==lowaddress[_n-1] 


gen my0=(year(lastsaledate)-1988)*12+month(lastsaledate)


sort idl numdatel
by idl: gen index=_n

*month number of current list price
gen my=(year(numdatel)-1988)*12+month(numdatel)
gen weeknum=numdatel-dow(numdatel)

**get variables that help adjust the list price 
gen seas=mod(my,12)
gen TOM2=TOM*TOM
gen num_lpchanges=min(lpchng,5)
gen chngprice=(num_lpchanges>1)
gen tenure=(numdatel-lastsaledate)/365
gen short=(TOM<=14)
gen medium=(TOM>14&TOM<45)
gen medium2=(TOM>=45&TOM<90)
gen longer=(TOM>=90& TOM<180)
gen expire=(TOM>=174 & TOM<=183)
gen short_chngprice=short*chngprice
gen medium_chngprice=medium*chngprice
gen longer_chngprice=(TOM>=90)*chngprice
gen soon1=chngprice*(lptime<=31)
gen soon2=chngprice*(lptime<=14)
gen indivs = (strpos(sr_seller, ",")~=0)
gen indivb = (strpos(sr_buyer, ",")~=0)
gen noarms=(sr_arms_length_flag!="1")
gen estate=(strpos(sr_buyer,"ESTATE")~=0)
gen family=(strpos(sr_buyer,"FAMILY")~=0)
gen living=(strpos(sr_buyer,"LIVING")~=0)
gen trust=(substr(sr_buyer,-2,2)=="TR")
gen relocat=(strpos(sr_buyer,"RELOC")~=0|strpos(sr_buyer,"MOBILITY")~=0)
replace trust=1 if substr(sr_buyer,-5,5)=="TRUST"
replace trust=1 if substr(sr_buyer,1,5)=="TRUST"
gen foreclosure2=0
replace foreclosure2=1 if strpos(sr_buyer,"BANK")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"BANC")~=0 & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer," BA ")~=0 & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"BK")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"JP")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"MORT")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"MTG")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"HUD-")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if sr_buyer=="HUD"  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LOA")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LNS")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LN")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SECU")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"ASSET")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"DEUTSCHE")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"STEARNS")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FARGO")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SERIE")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"STANLEY")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"MORGAN")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CITI")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"BOSTON")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"WAMU")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"MUTUAL")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LYNCH")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LEHMAN")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"REO ")~=0     & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer," SECS ")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FEDERAL")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FDIC")~=0     & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FRANKLIN")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"COUNTY")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CLERK")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SHERIFF")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"INVEST")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"INVS")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"DEFAULT")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FORECL")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"HOLDING")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SAVING")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LENDERS")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LENDING")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SACHS")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"DOWNEY")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FREMONT")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SUISSE")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CWALT")~=0    & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"GSAMP")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2003-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2004-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2005-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2006-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2007-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2008-")~=0  & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2009-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2010-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2011-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2012-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"2013-")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"TD")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SERVICE")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SERVICING")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"SVC")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FUNDING")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"HOMESALES")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0 
replace foreclosure2=1 if strpos(sr_buyer,"ABS")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LLC")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FINANCIAL")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CIT GROUP")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CIT GRP")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"LIQUIDATION")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FANNIE")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"FREDDIE")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"BENEFICIAL")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CALIFORNIA HOUSING")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"CALIFORNIA HSNG")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"DB STRUCTURED PRODUCTS INC")~=0   & indivb==0 & noarms==1 & family==0 & estate==0 & living==0  & relocat==0
replace foreclosure2=1 if strpos(sr_buyer,"REAL")~=0   & indivb==0 & noarms==1 & family==0  & living==0  & relocat==0
rename foreclosure2 foreclosure
replace foreclosure=0 if mergetype_f==0
gen longTOM=(TOM>180)*TOM
gen longTOM2=(TOM>180)*TOM2

gen raise=(ratio>1)
gen ratio_chngprice=ratio*chngprice
gen above=(ln(lastsalepr)-ln(price)>0)

gen interval=my-my0
drop if interval<6 
gen temp=abs(ln(price)-ln(lastsalepr))/interval
drop if temp>.05
drop if price<20000
drop if lastsalepr<20000
drop if price>4000000
drop if lastsalepr>4000000
drop if my0<thresh
keep if ratio<1.25 & ratio>.65
drop temp
gen temp=abs(ln(price)-ln(lastsalepr))/interval

sort numdatel

pctile pct=temp ,nq(100)
gen rw=1
replace rw=.25 if temp>pct[95]
replace rw=.75 if temp>pct[90] & temp<=pct[95]
*this is taken from CS methodology on interval weights 
gen rw2=-.02105*interval/12+1.0105
drop temp

gen dayl=day(numdatel)
gen monthl=month(numdatel)
gen yearl=year(numdatel)

*more variables that are helpful for adjusting list price
gen temp=string(price)
gen digit=substr(temp,-4,1)
destring digit,replace
gen precise=(digit==1|digit==2|digit==3|digit==6|digit==7|digit==8)
gen justbelow=(digit==4|digit==9)
gen digit2=substr(temp,-3,1)
gen digit3=substr(temp,-5,1)
gen digit4=substr(temp,-6,1)
gen hundreds=(digit2!="0")
gen tenthou=(digit3!="0")
destring digit3 digit,replace
gen t1=(digit3==5&digit==0)
gen t2=(digit3==9&digit==9)
gen t3=(digit3==9&digit==5)
gen popular=min(t1+t2+t3,1)

gen listdate=numdatel-TOM

gen wedge2=ln(salepr)-ln(price)
gen wedge3=salepr/price
replace wedge3=. if mergetype_c!=1
replace wedge2=. if mergetype_c!=1

sort lowaddress numdatel
gen m1=(length_new[_n-1]<=4 & lowaddress==lowaddress[_n-1])
gen m2=(length_new[_n-1]>4 & length_new[_n-1]<=26 & lowaddress==lowaddress[_n-1])

gen q=1
replace q=2 if month(numdatel)>=4
replace q=3 if month(numdatel)>=7
replace q=4 if month(numdatel)>=10

save txyz,replace
collapse (mean) wedge3,by(my)
sort my
rename wedge3 lagmwedge3
save mergewedge3,replace

use txyz,clear

**predict propensity to sell
xi: dprobit mergetype_c  popular hundreds tenthou precise justbelow TOM TOM2 longTOM longTOM2 ratio_chngprice raise chngprice above foreclosure longer_chngprice soon1 expire m1 m2 i.seas if     listdate>17535 & yearl<2012,r
predict shat
predict shat2,xb
gen invmills=normalden(shat2)/normal(shat2)
drop shat2


**predict sale to list price ratio
gen myt=my
forvalues z=2/8{
replace my=myt-`z'
sort my
merge my using mergewedge3,nokeep
tab _merge
drop _merge
rename lagmwedge3 lagmwedge3_`z'
xi: reg wedge3 invmills popular hundreds tenthou precise justbelow TOM TOM2 longTOM longTOM2 ratio_chngprice raise chngprice above foreclosure m1 m2 lagmwedge3_`z' i.seas if wedge2<.8 & wedge2>=-.5 & mergetype_c==1 & listdate>17535 & my<=297 ,r
predict ww`z'
}
rm txyz.dta
replace my=myt
drop myt


gen rw3=shat
forvalues z=2/8{
gen adjprice`z'=ww`z'*price/10000
replace adjprice`z'=price/10000 if adjprice`z'==.
replace adjprice`z'=adjprice2
}
*gen adjprice=price*ww
replace rw3=1 if rw3==.

drop if ww2==.

gen id=_n

replace price=price/10000
replace lastsalepr=lastsalepr/10000
egen houseid=group(lowaddress)
sort houseid weeknum


drop temp*
**hold over from old code
replace datediff_c=-1*datediff_c

**closing date distribution to use for simulated rps index
gen weights_0=0
forvalues a=-4/52{
gen temp=0
qui replace temp= 1 if datediff_c>-7*`a' & datediff_c<=-7*(`a'-1) & mergetype_c==1 & datedifftemp==1  
egen temp2=sum(temp)
local aa=`a'+5
qui replace weights_0=temp2 in `aa'
drop temp*
}
forvalues b=1/11{
gen weights_`b'=weights_0
}


keep  dayl my my0 lastsalepr price adjprice2 adjprice3 adjprice4 adjprice5 adjprice6 adjprice7 adjprice8 rw rw2 rw3 seas houseid weeknum  weights*
order  dayl my my0 lastsalepr price adjprice2 adjprice3 adjprice4 adjprice5 adjprice6 adjprice7 adjprice8 rw rw2 rw3 seas houseid weeknum weights*
outsheet using `j'_list.txt,nonames replace
capture log close
}


